{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ifmV02zKlsCs",
        "outputId": "06fd2b0a-209a-465f-a28d-e5127ef0f8bf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m493.7/493.7 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m7.9/7.9 MB\u001b[0m \u001b[31m90.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m311.2/311.2 kB\u001b[0m \u001b[31m39.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m124.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m84.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m39.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install -qU \\\n",
        "  datasets==2.14.6 \\\n",
        "  transformers==4.35.0"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q4vTJ-pFmWl5"
      },
      "source": [
        "## Dataset Download"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GFaaDw5VmZEk"
      },
      "source": [
        "We're going to test with a more real world use-case, with messy, imperfect data. We will use the [`jamescalam/ai-arxiv-chunked`](https://huggingface.co/datasets/jamescalam/ai-arxiv-chunked) dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 234,
          "referenced_widgets": [
            "7ea01bde70bb45e191bd17c930432630",
            "f3806a03345f44c59d3af1dd1b65a9a1",
            "77498407a21449bfbd5bd173d12edebc",
            "f9dd384b38764391ba6cb40c75798616",
            "70fb65dbd57544f5a0c6a6b5867c4462",
            "520ba318cd5546ac98b719910c6c97a2",
            "28482a116f744e17b6455a832ac80ce4",
            "1d84ee9d778340aeb6db2091e8f459ee",
            "5358dd1b95e34e5ba0e3dc86aee64eac",
            "0b66dcc0dc4d4edbbb3a0772c9277c56",
            "17581778d8af4a31b6a6268fafad99a8",
            "95b73396a6124c2687591067535ce303",
            "edb190531ef747a3a7d87d24079e2ea5",
            "1f5e0b297de84e128d654ece0a439730",
            "19e004da2ff54a67b421720841d41238",
            "66afb906d3e244639c5c63315ac7506c",
            "3435ea75934a48a2b52ae2a2b38c0335",
            "78cfd81d0d4a49788752da8c4dd27862",
            "9a13a41f652b4a53b2507415eab91b7e",
            "df5dbd8118d14a00a70d1fb9eeff9bb4",
            "9d825dd6d6ac4907b650ba47012d0798",
            "7f58cd1d5af44dff8f078ad3607fee34",
            "eb4f77b14ec1422da5a5fc3886ab669e",
            "198cdc09b9dd421fb5aae2f58312c1af",
            "cad0a3bedcdc421db342d5975f309872",
            "1538e0eb8c594cf8a47eaab21ae32558",
            "6dd7b2904dd6427ba3a5dbcafbe99202",
            "b08599d670e1412ca9f8f344ae085a73",
            "b01badd940ed4b809d2564782fc02942",
            "01fc5a7e348349ce8f184db5a10a9562",
            "b37d721a596244099dcfb68e7d7608b6",
            "ed4cff954c734ee29a51fe5b3db6bd40",
            "8a36e42851354d6f9604f15580f0cec0",
            "d70c70d0cc19436382de189051f109d5",
            "6f993ad492aa4976b664421c21104ffa",
            "0cdad1ce3cfd47869664098d239c11d7",
            "522b618c2da0465e9dde48970fe7c1a7",
            "a750896fc64e4746aec35397226be70d",
            "bc767bd783e9462ca91ed6623631eb85",
            "67b65dbd548c4ffb9eac381f0486ea51",
            "4e6fa239036c4578b05cd379eadb99e3",
            "d06cbc1470c942c093b3e0541a906272",
            "e0ea87aeaa164942af03fcf56abbda8a",
            "648e57d888ea487ba9095423b5ed131a"
          ]
        },
        "id": "4-FqcdKHmVpa",
        "outputId": "a69ec96d-e0f1-4d79-feb4-e4db1cd41c89"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7ea01bde70bb45e191bd17c930432630"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/153M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "95b73396a6124c2687591067535ce303"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "eb4f77b14ec1422da5a5fc3886ab669e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d70c70d0cc19436382de189051f109d5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['doi', 'chunk-id', 'chunk', 'id', 'title', 'summary', 'source', 'authors', 'categories', 'comment', 'journal_ref', 'primary_category', 'published', 'updated', 'references'],\n",
              "    num_rows: 41584\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "data = load_dataset(\"jamescalam/ai-arxiv-chunked\", split=\"train\")\n",
        "data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gp5a_bInyfdX"
      },
      "source": [
        "First we define our embedding function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "oG6zd1dLw54w",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 227,
          "referenced_widgets": [
            "9163a498d987420d8d57d30dee9ce3b0",
            "a26ac21fcfd44c47be19b0d253f064e6",
            "a078f05b76304ca19974f44716891df3",
            "5355d550df7b47829b4727a5ae5283cc",
            "6b32dda476c14595bbd7d5d8056c2db0",
            "dd645e076f0440eea1f0a80f832c61b8",
            "024ae3382fbe41ca84897a9159ec1a71",
            "a922e02430d6410b93552d49458b9080",
            "09cccd2e8ef543ce83a768a288fe8e61",
            "5670cafd00ab45deaabed087e6b6b7d6",
            "cb3764443420440983280b6154a453f9",
            "fc8a4c8ea63041bbb95a9d871f457af5",
            "6bbaf8fe42b64790a66177169497524a",
            "113eba16f2db4e6ea432805236301c6c",
            "e287b284e051469fbac25ab5b9d1ffc6",
            "da733bf5f93547fd87612a01e98855d3",
            "7ac2d94346da477e8818b7092ba6a0e3",
            "7211a3484875419c8684ef4603e12f45",
            "227c8e8452354f6dabf412cebf88551d",
            "3e34642c0b0f47a2853497a9f8c8fe30",
            "cdcdabf93a2e45e9911f7094371c363a",
            "b412660754e9489c9da6981178252369",
            "e0fd643ae3994a2ebcbb197fc6d86b1f",
            "abc7039866fd4ce9b0f5c35ca795b2fd",
            "2ed75ad9b1a24728a07b516b1c63ccbb",
            "1b1812c174ba438fbdf29e26f172a2b7",
            "724c77ce1744490bac73dec9482024f3",
            "d44473a8177d49c6a9020bd0045276ba",
            "64c9e05c968d46a0b4d92c82e0ba2066",
            "0c8aea3115834d9c8536abc3518de042",
            "20e8e4f6e9a849279bfdb5a0f3c36d1e",
            "aa0218ac393846959b7f687207b98e6b",
            "4cf9b53c8ad3415392239436585b4d8b",
            "ff87f8355bba485e8e97509c3cbdb2e4",
            "90573c38aba1488ebb21686acc2b71c1",
            "05bb3c0e20f3418682dec839b1df4083",
            "dd871f8a47214a7f8bd54fd0b8dfb9e8",
            "ac02291e0b8449ec964cafe178af41f3",
            "08c2e2873dc24788b9723b3053257f54",
            "9c7cb28930b54947a0ec3c1ecb99afd0",
            "4958813e096c4b838f3e132db1fd7418",
            "38eb3c8a467c44659105234d718d9dd6",
            "62641a7191a942948f27584fe933985a",
            "8461d68b8455458a9a26bb86c5d26e76",
            "94bb4a00f19a4e11860b98861cd5651f",
            "deda98d0247546288c8e0182dc022730",
            "10333289efa442b2b782334a47d81f55",
            "8ef369fddf0f4366b7214be58ca4940e",
            "833a911eba784a39a18e35657b6a35c2",
            "e2732dcd49f14857a34262e9ccf2243e",
            "c919064e6ba843a4b2117f024c958fcb",
            "c4e8dda1c528453ba5ffa25191858646",
            "fc6dc4aa002e4b84afcdcf1b568b63cb",
            "d2c061b14d9e4a3f95945cf97bcd39e6",
            "aff6c77f8a804127b0d1be9f8fb65a2e",
            "061d8d15996f44348625ac854447924b",
            "190dc644dd77412d85262618a6881ed9",
            "784bb97ad3114f61a4b590cabf874c4d",
            "c3d5240773cd4e9d97af36deddf7858e",
            "b610a4e4165b4d3ea9c5c1abf04a384f",
            "35d13651a6d04d12b78cff9027ff686c",
            "322d567a268d4ea79bd8c682de4d1f9b",
            "71af8edff769470293ff2e35a4ad6a03",
            "8e65f88a19b7415dbd00e7fd0f514943",
            "c498e5edea1d429aab0dade173af698e",
            "df26fba096034564a58eb0569d8f65e9"
          ]
        },
        "outputId": "9dcbbbff-ba8a-43b1-b115-ec3a529f4b48"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cuda\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (\u2026)okenizer_config.json:   0%|          | 0.00/342 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9163a498d987420d8d57d30dee9ce3b0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (\u2026)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fc8a4c8ea63041bbb95a9d871f457af5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (\u2026)/main/tokenizer.json:   0%|          | 0.00/712k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e0fd643ae3994a2ebcbb197fc6d86b1f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (\u2026)cial_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ff87f8355bba485e8e97509c3cbdb2e4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (\u2026)lve/main/config.json:   0%|          | 0.00/619 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "94bb4a00f19a4e11860b98861cd5651f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/670M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "061d8d15996f44348625ac854447924b"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "import torch\n",
        "from torch.nn.functional import normalize\n",
        "from transformers import AutoModel, AutoTokenizer\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "print(f\"Using {device}\")\n",
        "\n",
        "model_id = \"thenlper/gte-large\"\n",
        "\n",
        "# initialize tokenizer and model\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
        "model = AutoModel.from_pretrained(model_id).to(device)\n",
        "model.eval()\n",
        "\n",
        "def embed(docs: list[str]) -> list[list[float]]:\n",
        "    # tokenize\n",
        "    tokens = tokenizer(\n",
        "        docs, padding=True, max_length=512, truncation=True, return_tensors=\"pt\"\n",
        "    ).to(device)\n",
        "    with torch.no_grad():\n",
        "        # process with model for token-level embeddings\n",
        "        out = model(**tokens)\n",
        "        # mask padding tokens\n",
        "        last_hidden = out.last_hidden_state.masked_fill(\n",
        "            ~tokens[\"attention_mask\"][..., None].bool(), 0.0\n",
        "        )\n",
        "        # create mean pooled embeddings\n",
        "        doc_embeds = last_hidden.sum(dim=1) / \\\n",
        "            tokens[\"attention_mask\"].sum(dim=1)[..., None]\n",
        "    return doc_embeds.cpu().numpy()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1nvrNQSGXvEC"
      },
      "source": [
        "Use this to build a Numpy array of cohere embedding vectors."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "EdyWVR17zX7I",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "0f448a8428494784979bc6ed387185ce",
            "806b59eaf7f24ab7a431afdf3188c425",
            "6ec34bc9273a4782a3cfdcf7276cfe33",
            "b763d4e20ff047e290a79db6e752f9f6",
            "19b6946fb7c342cbac322bd525f879b9",
            "dfc5e7a73b2c41b88747d42929b084a6",
            "5b2c7154b20049bd903eb18204d1b8bd",
            "9ee854c8377e45378d15545b6f4b2afd",
            "4cfc8a59f8194c3d8940022a1db8473f",
            "cbcd3f7dc64b454095f43dc9c84feb32",
            "d3cd379822e34725b0416fe12b2ece12"
          ]
        },
        "outputId": "107dc5e4-4a48-4cba-cdb0-4480c8f8939f"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/325 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0f448a8428494784979bc6ed387185ce"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from tqdm.auto import tqdm\n",
        "import numpy as np\n",
        "\n",
        "chunks = data[\"chunk\"]\n",
        "batch_size = 128\n",
        "\n",
        "for i in tqdm(range(0, len(chunks), batch_size)):\n",
        "    i_end = min(len(chunks), i+batch_size)\n",
        "    chunk_batch = chunks[i:i_end]\n",
        "    # embed current batch\n",
        "    embed_batch = embed(chunk_batch)\n",
        "    # add to existing np array if exists (otherwise create)\n",
        "    if i == 0:\n",
        "        arr = embed_batch.copy()\n",
        "    else:\n",
        "        arr = np.concatenate([arr, embed_batch.copy()])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bl9g3ePt029u"
      },
      "source": [
        "Now we need to create the query mechanism, this is simply a cosine similarity calculation between a query vector and our `arr` vectors."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "MR7WyDiatlsX"
      },
      "outputs": [],
      "source": [
        "from numpy.linalg import norm\n",
        "\n",
        "# convert chunks list to array for easy indexing\n",
        "chunk_arr = np.array(chunks)\n",
        "\n",
        "def query(text: str, top_k: int=3) -> list[str]:\n",
        "    # create query embedding\n",
        "    xq = embed([text])[0]\n",
        "    # calculate cosine similarities\n",
        "    sim = np.dot(arr, xq.T) / (norm(arr, axis=1)*norm(xq.T))\n",
        "    # get indices of top_k records\n",
        "    idx = np.argpartition(sim, -top_k)[-top_k:]\n",
        "    docs = chunk_arr[idx]\n",
        "    for d in docs.tolist():\n",
        "        print(d)\n",
        "        print(\"----------\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u2NjYxsn7J5f",
        "outputId": "5768eb41-2680-407e-ba09-3d89f3ca8ace"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u2022Zero-shot. We provide a textual description\n",
            "of the task and a test example. The model\n",
            "either provides an answer using open-ended\n",
            "generation, or ranks the proposed answers.\n",
            "\u2022Few-shot. We provide a few examples of the\n",
            "task (between 1 and 64) and a test example.\n",
            "The model takes this text as input and generates the answer or ranks different options.\n",
            "We compare LLaMA with other foundation models, namely the non-publicly available language\n",
            "models GPT-3 (Brown et al., 2020), Gopher (Rae\n",
            "et al., 2021), Chinchilla (Hoffmann et al., 2022)\n",
            "and PaLM (Chowdhery et al., 2022), as well as\n",
            "the open-sourced OPT models (Zhang et al., 2022),\n",
            "GPT-J (Wang and Komatsuzaki, 2021), and GPTNeo (Black et al., 2022). In Section 4, we also\n",
            "brie\ufb02y compare LLaMA with instruction-tuned\n",
            "models such as OPT-IML (Iyer et al., 2022) and\n",
            "Flan-PaLM (Chung et al., 2022).We evaluate LLaMA on free-form generation\n",
            "tasks and multiple choice tasks. In the multiple\n",
            "choice tasks, the objective is to select the most\n",
            "----------\n",
            "but BoolQ. Similarly, this model surpasses PaLM540B everywhere but on BoolQ and WinoGrande.\n",
            "LLaMA-13B model also outperforms GPT-3 on\n",
            "most benchmarks despite being 10 \u0002smaller.\n",
            "3.2 Closed-book Question Answering\n",
            "We compare LLaMA to existing large language\n",
            "models on two closed-book question answering\n",
            "benchmarks: Natural Questions (Kwiatkowski\n",
            "et al., 2019) and TriviaQA (Joshi et al., 2017). For\n",
            "both benchmarks, we report exact match performance in a closed book setting, i.e., where the models do not have access to documents that contain\n",
            "evidence to answer the question. In Table 4, we\n",
            "report performance on NaturalQuestions, and in Table 5, we report on TriviaQA. On both benchmarks,\n",
            "LLaMA-65B achieve state-of-the-arts performance\n",
            "in the zero-shot and few-shot settings. More importantly, the LLaMA-13B is also competitive on\n",
            "these benchmarks with GPT-3 and Chinchilla, despite being 5-10 \u0002smaller. This model runs on a\n",
            "single V100 GPU during inference.\n",
            "0-shot 1-shot 5-shot 64-shot\n",
            "Gopher 280B 43.5 - 57.0 57.2\n",
            "----------\n",
            "31.39%LLaMA-GPT4 \n",
            " 25.99%\n",
            "Tie \n",
            " 42.61%\n",
            "HonestyAlpaca \n",
            "  25.43%LLaMA-GPT4 \n",
            " 16.48%\n",
            "Tie \n",
            " 58.10%\n",
            "Harmlessness(a) LLaMA-GPT4 vs Alpaca ( i.e.,LLaMA-GPT3 )\n",
            " GPT4 \n",
            "  44.11%\n",
            "LLaMA-GPT4 \n",
            " 42.78% Tie \n",
            " 13.11%\n",
            "Helpfulness GPT4 \n",
            "  37.48%\n",
            "LLaMA-GPT4 \n",
            " 37.88% Tie \n",
            " 24.64%\n",
            "Honesty GPT4 \n",
            "  35.36% LLaMA-GPT4 \n",
            " 31.66% Tie \n",
            " 32.98%\n",
            "Harmlessness\n",
            "(b) LLaMA-GPT4 vs GPT-4\n",
            "Figure 3: Human evaluation.\n",
            "4.2 H UMAN EVALUATION WITH ALIGNMENT CRITERIA\n",
            "To evaluate the alignment quality of our instruction-tuned LLMs, we follow alignment criteria from\n",
            "Anthropic Askell et al. (2021): an assistant is aligned if it is helpful, honest, and harmless (HHH).\n",
            "----------\n"
          ]
        }
      ],
      "source": [
        "query(\"why should I use llama 2?\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "ZSIBTMUy7qc0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "df4606f5-ba73-4c90-9656-a110ad34515b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "future work, we plan on explicitly comparing and contrasting (semi-)manual versus automated approaches\n",
            "to red teaming in order to determine how the two methods vary in the ef\ufb01cacy and diversity of resulting red\n",
            "team attacks.\n",
            "5.2 Policy Interventions\n",
            "Red teaming entails working with inherently controversial subject matter, and most organizations that red\n",
            "team systems have strong counter-incentives to share their \ufb01ndings.13This is a problem; if we cannot publicly\n",
            "13Red team datasets include offensive content, and may potentially reveal embarrassing or sensitive details about an\n",
            "institution\u2019s AI system if released publicly.\n",
            "14\n",
            "discuss \u2014 in detail \u2014 how we red team systems and what we learn as a result, it will be dif\ufb01cult to broadly\n",
            "share the future risks, failures, and implications of yet-to-be developed systems. This problem gets worse\n",
            "over time. As systems become more capable, the results of red teaming may surface increasingly undesirable\n",
            "harms. Therefore, we need to change the incentive structure so more organizations share \ufb01ndings from their\n",
            "red teaming efforts when doing so is safe and bene\ufb01cial. To do so, we identify two speci\ufb01c interventions the\n",
            "AI research community could take to build consensus around how to red team andhow to release \ufb01ndings\n",
            "from red teaming .\n",
            "----------\n",
            "more comprehensive way.\n",
            "We conducted a series of red teaming with various groups of internal employees, contract workers, and\n",
            "externalvendors. Theseteamsincludedover350people,includingdomainexpertsincybersecurity,election fraud, social media misinformation, legal, policy, civil rights, ethics, software engineering, machine\n",
            "learning, responsible AI, and creative writing. They also included individuals representative of a variety of\n",
            "socioeconomic, gender, ethnicity, and racial demographics.\n",
            "28\n",
            "Theredteamersprobedourmodelsacrossawiderangeofriskcategories(suchascriminalplanning,human\n",
            "tra\ufb03cking, regulated or controlled substances, sexually explicit content, unquali\ufb01ed health or \ufb01nancial\n",
            "advice, privacy violations, and more), as well as di\ufb00erent attack vectors (such as hypothetical questions,\n",
            "malformed/misspelledinputs,orextendeddialogues). Additionally,weconductedspeci\ufb01cteststodetermine\n",
            "the capabilities of our models to facilitate the production of weapons (e.g. nuclear, biological, chemical, and\n",
            "cyber); \ufb01ndingsonthesetopicsweremarginal andweremitigated. Nonetheless, wewill continueourred\n",
            "teaming e\ufb00orts in this front.\n",
            "----------\n",
            "AI research community could take to build consensus around how to red team andhow to release \ufb01ndings\n",
            "from red teaming .\n",
            "For how to red team , we have detailed our initial approach. However, we conducted this effort in isolation, and we would have bene\ufb01ted from participating in a community-based effort to address certain open\n",
            "questions:\n",
            "\u2022 Who should red team and why?\n",
            "\u2022 What protections should we put in place to ensure the safety of the red team?\n",
            "\u2022 What instructions and information about the models should we provide to the red team?\n",
            "\u2022 How should we annotate and analyze the data we collect?\n",
            "\u2022 What constitutes a successful red team attempt?\n",
            "We can make progress towards answering these questions by convening a multidisciplinary community to\n",
            "share different approaches to internal red teaming and drive toward consensus.\n",
            "The research community lacks shared norms and best practices for how to release \ufb01ndings from red teaming. As a result, we made our decision to release the data largely on our own and likely missed critical\n",
            "perspectives from experts, other disciplines, and members of the public.14The decision for how to appropriately release \ufb01ndings will ultimately require a subjective judgment call. For our purposes, we reviewed a\n",
            "sample of our red team dataset and evaluated the pros and cons of a public release (See \u00a7A.5). Among them\n",
            "----------\n"
          ]
        }
      ],
      "source": [
        "query(\"can you tell me about red teaming for llama 2?\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query(\"what is the best llm?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2VUqwF-com3z",
        "outputId": "a66b4aa6-1f85-4de1-e4eb-9a6aa67fdc71"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "faces negligible performance degradation compared to its uncompressed original, while it consumes\n",
            "only 25% of the GPU memory required by the uncompressed version, thus supporting its effective\n",
            "inference on 4\u0002RTX 3090 Ti (24G) or 8 \u0002RTX 2080 Ti (11G). We will attempt to further reduce\n",
            "the resource requirements and keep the community updated on this important working item.\n",
            "23\n",
            "Technical Report 2022-10-06 (v1)\n",
            "B E THICS : EVALUATION ON BIASES AND TOXICITY\n",
            "Albeit LLMs\u2019 strong abilities in language and beyond, which could bring substantial welfare to\n",
            "human beings, they can potentially produce toxic and illegal contents for evil use (Weidinger et al.,\n",
            "2021; Sheng et al., 2021; Dev et al., 2021; Bommasani et al., 2021). In GLM-130B, before granting\n",
            "model weight to applicants, in the model license, we demand them to agree that they will not use it\n",
            "for any deeds that may be harmful to society and human beings.\n",
            "Additionally, from a technical perspective, we argue that we must also understand LLMs\u2019 toxic\n",
            "and biased behaviors and ultimately eliminate them. This aligns with our commitment to \u201cLLM\n",
            "Inclusivity\u201d, as it is necessary to include more people in the open-sourced LLM research to facilitate\n",
            "----------\n",
            "thinking, problem-solving, and analytical skills, making them ideal for evaluating the performance\n",
            "of large language models in relation to human cognition. More specifically, we collect exams\n",
            "corresponding to 8 subjects from Chinese Gaokao: history, math, English, Chinese, geography,\n",
            "biology, chemistry and physics. We select mathematical questions from GRE, select English and\n",
            "math subjects from SAT to construct the benchmark.\n",
            "Law School Admission Test: Law school admission tests, such as the LSAT , are intended to measure\n",
            "the reasoning and analytical skills of prospective law students. These tests include sections on logical\n",
            "reasoning, reading comprehension, and analytical reasoning, which challenge the test-takers\u2019 ability\n",
            "to analyze complex information and draw accurate conclusions. Incorporating these tasks in our\n",
            "benchmark enables us to assess language models\u2019 capabilities in legal reasoning and analysis.\n",
            "Lawyer Qualification Test: Lawyer qualification tests, such as the bar exam, assess the legal\n",
            "knowledge, analytical skills, and ethical understanding of individuals pursuing a career in law. These\n",
            "exams cover a broad range of legal topics, including constitutional law, contract law, criminal law, and\n",
            "property law, and require candidates to demonstrate their ability to apply legal principles and reason\n",
            "effectively. By incorporating lawyer qualification tests in our benchmark, we can evaluate language\n",
            "models\u2019 performance in the context of professional legal expertise and ethical judgment. Specifically,\n",
            "----------\n",
            "effectively. By incorporating lawyer qualification tests in our benchmark, we can evaluate language\n",
            "models\u2019 performance in the context of professional legal expertise and ethical judgment. Specifically,\n",
            "we select questions from previous Chinese lawyer qualification tests to build the benchmark.\n",
            "Graduate Management Admission Test (GMAT) : The GMAT is a standardized exam designed\n",
            "to assess the analytical, quantitative, verbal, and integrated reasoning skills of prospective graduate\n",
            "business school students. The GMAT consists of sections such as Analytical Writing Assessment,\n",
            "Integrated Reasoning, Quantitative Reasoning, and Verbal Reasoning, which evaluate the test-takers\u2019\n",
            "abilities to think critically, analyze data, and communicate effectively. Incorporating GMAT tasks in\n",
            "our benchmark enables us to assess the language models\u2019 performance in a business context and their\n",
            "potential to assist in decision-making and problem-solving in management scenarios.\n",
            "High School Math Competitions: High school math competitions, such as the American Mathematics Competitions (AMC) and the American Invitational Mathematics Examination (AIME) ,\n",
            "challenge students\u2019 mathematical abilities, creativity, and problem-solving skills. These contests\n",
            "cover a wide range of mathematical topics, including number theory, algebra, geometry, and combinatorics, and often feature unconventional problems that require inventive approaches to solve. By\n",
            "incorporating tasks from high school math competitions into our benchmark, we can further evaluate\n",
            "----------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query(\"what is the difference between gpt-4 and llama 2?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sF8xvtyaoom4",
        "outputId": "742284d7-0da3-41a1-b6a3-45cdc9c8b26d"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "-0.043\n",
            "-0.009+0.0132-0.004 +0.0562\n",
            "+0.0387-0.012\n",
            "-0.076Alpaca: 0.39 LLaMA-GPT4: 0.34 GPT4: 0.37Figure 6: ROUGE-L on unnatural instructions evaluated with 9K samples. The instructions are\n",
            "grouped into four subsets based on the ground-truth response length. The mean values are reported in\n",
            "the legend. The difference with GPT-4 is reported on the bar per group. LLaMA-GPT4 is a closer\n",
            "proxy to GPT-4 than Alpaca.\n",
            "closely follow the behavior of GPT-4. When the sequence length is short, both LLaMA-GPT4 and\n",
            "GPT-4 can generate responses that contains the simple ground truth answers, but add extra words to\n",
            "make the response more chat-like, which probably leads to lower ROUGE-L scores.\n",
            "5 R ELATED WORK\n",
            "Instruction Tuning. Instruction tuning of LLMs is an increasingly popular research direction in\n",
            "NLP (Zhong et al., 2021; Ouyang et al., 2022; Wei et al., 2021). Existing works aim to improve\n",
            "the quality and scale of three factors in the development pipeline, including instruction-following\n",
            "----------\n",
            "to GPT-3 corresponds to the Stanford Alpaca model. From Figure 3(a), we observe that ( i) For the\n",
            "\u201cHelpfulness\u201d criterion, GPT-4 is the clear winner with 54.12% of the votes. GPT-3 only wins 19.74%\n",
            "of the time. ( ii) For the \u201cHonesty\u201d and \u201cHarmlessness\u201d criteria, the largest portion of votes goes\n",
            "to the tie category, which is substantially higher than the winning categories but GPT-3 (Alpaca) is\n",
            "slightly superior.\n",
            "Second, we compare GPT-4-instruction-tuned LLaMA models against the teacher model GPT-4 in\n",
            "Figure 3(b). The observations are quite consistent over the three criteria: GPT-4-instruction-tuned\n",
            "LLaMA performs similarly to the original GPT-4. We conclude that learning from GPT-4 generated\n",
            "5\n",
            "60% 70% 80% 90% 100%12345BRanking Group 94% 624 : 66792% 614 : 67091% 623 : 68289% 597 : 66989% 605 : 67891% 609 : 666\n",
            "----------\n",
            "31.39%LLaMA-GPT4 \n",
            " 25.99%\n",
            "Tie \n",
            " 42.61%\n",
            "HonestyAlpaca \n",
            "  25.43%LLaMA-GPT4 \n",
            " 16.48%\n",
            "Tie \n",
            " 58.10%\n",
            "Harmlessness(a) LLaMA-GPT4 vs Alpaca ( i.e.,LLaMA-GPT3 )\n",
            " GPT4 \n",
            "  44.11%\n",
            "LLaMA-GPT4 \n",
            " 42.78% Tie \n",
            " 13.11%\n",
            "Helpfulness GPT4 \n",
            "  37.48%\n",
            "LLaMA-GPT4 \n",
            " 37.88% Tie \n",
            " 24.64%\n",
            "Honesty GPT4 \n",
            "  35.36% LLaMA-GPT4 \n",
            " 31.66% Tie \n",
            " 32.98%\n",
            "Harmlessness\n",
            "(b) LLaMA-GPT4 vs GPT-4\n",
            "Figure 3: Human evaluation.\n",
            "4.2 H UMAN EVALUATION WITH ALIGNMENT CRITERIA\n",
            "To evaluate the alignment quality of our instruction-tuned LLMs, we follow alignment criteria from\n",
            "Anthropic Askell et al. (2021): an assistant is aligned if it is helpful, honest, and harmless (HHH).\n",
            "----------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---"
      ],
      "metadata": {
        "id": "d2UP-QJzo610"
      }
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "gpuType": "V100"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}